import requests
from bs4 import BeautifulSoup
import re

book_names = list()
book_prices = list()
book_avail = list()
book_desc = list()
book_type = list()
book_rating = list()
book_img = list()
start_url = 'http://books.toscrape.com/catalogue/category/books_1/'
start_page = 'page-1.html'
next_page_url = start_url + start_page

while next_page_url:
    print("scrape ", next_page_url)
    r = requests.get(next_page_url)
    r.raise_for_status()
    bs = BeautifulSoup(r.text, 'html.parser')
    for div in list(bs.find_all('div', class_='image_container')):
        # print(list(div.children)[1], list(div.children)[1].name, list(div.children)[1]['href'])
        page2 = list(div.children)[1]['href']
        r2 = requests.get(start_url + page2)
        r2.raise_for_status()
        bs2 = BeautifulSoup(r2.text, 'html.parser')
        book_prices.append(bs2.find('p', class_='price_color').contents[0])
        book_avail.append(bs2.find('p', class_='instock availability').contents[-1].strip())
        if bs2.find('div', id='product_description'):
            book_desc.append(bs2.find('div', id='product_description').next_sibling.next_sibling.contents[0])
        else:
            book_desc.append('')
        name = bs2.find('li', class_='active').contents[0]
        book_names.append(name)
        book_type.append(bs2.find('li', class_='active').previous_sibling.previous_sibling.contents[1].contents[0])
        book_rating.append(bs2.find(class_=re.compile("^star-rating"))['class'][1])
        book_img.append(start_url+'../'+bs2.find('img', alt=name)['src'])
    next_page = bs.find('li', class_='next')
    if not next_page:
        break
    next_page_url = next_page.a['href']
    next_page_url = start_url + next_page_url
